{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://huseinhouse-storage.s3-ap-southeast-1.amazonaws.com/bert-bahasa/dumping-twitter-6-july-2019.json\n",
    "# !wget https://malaya-dataset.s3-ap-southeast-1.amazonaws.com/dumping/twitter/2020-02-22-twitter-dump-in.json\n",
    "# !wget https://malaya-dataset.s3-ap-southeast-1.amazonaws.com/dumping/twitter/2020-03-08-twitter-dump-in.json\n",
    "# !wget https://huseinhouse-storage.s3-ap-southeast-1.amazonaws.com/bert-bahasa/dumping-instagram-6-july-2019.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "695571"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('dumping-instagram-6-july-2019.json') as fopen:\n",
    "    instagram = json.load(fopen)\n",
    "    \n",
    "len(instagram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6597867"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('dumping-twitter-6-july-2019.json') as fopen:\n",
    "    twitter = json.load(fopen)\n",
    "    \n",
    "len(twitter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-22-twitter-dump-in.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1617795/1617795 [00:01<00:00, 1419559.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-08-twitter-dump-in.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1711555/1711555 [00:01<00:00, 1454402.12it/s]\n"
     ]
    }
   ],
   "source": [
    "files = ['2020-02-22-twitter-dump-in.json', '2020-03-08-twitter-dump-in.json']\n",
    "\n",
    "for file in files:\n",
    "    print(file)\n",
    "\n",
    "    with open(file) as fopen:\n",
    "        temp = json.load(fopen)\n",
    "\n",
    "    for i in tqdm(range(len(temp))):\n",
    "        retweet = temp[i]['retweet_text_full']\n",
    "        t = temp[i]['data_text']\n",
    "        if retweet != 'NULL' and len(retweet) > len(t):\n",
    "            t = retweet\n",
    "        twitter.append(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter = twitter + instagram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def preprocessing(string):\n",
    "    string = re.sub(\n",
    "        'http\\S+|www.\\S+',\n",
    "        '',\n",
    "        ' '.join(\n",
    "            [i for i in string.split() if i.find('#') < 0 and i.find('@') < 0]\n",
    "        ),\n",
    "    )\n",
    "    \n",
    "    chars = ',.()!:\\'\"/;=-'\n",
    "    for c in chars:\n",
    "        string = string.replace(c, f' {c} ')\n",
    "        \n",
    "    string = re.sub(\n",
    "        u'[0-9!@#$%^&*()_\\-+{}|\\~`\\'\";:?/.>,<]',\n",
    "        ' ',\n",
    "        string,\n",
    "        flags = re.UNICODE,\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    \n",
    "    return string.lower()\n",
    "\n",
    "def loop(strings):\n",
    "    for i in tqdm(range(len(strings))):\n",
    "        strings[i] = preprocessing(strings[i])\n",
    "    return strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:13<00:00, 50891.79it/s]\n",
      "100%|██████████| 663924/663924 [00:12<00:00, 52266.57it/s]\n",
      "100%|██████████| 663924/663924 [00:13<00:00, 50925.13it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 3022.92it/s]360.60it/s]\n",
      "100%|██████████| 663924/663924 [00:13<00:00, 50961.34it/s]\n",
      "100%|██████████| 663924/663924 [00:13<00:00, 49526.98it/s]\n",
      "100%|██████████| 663924/663924 [00:13<00:00, 50320.18it/s]\n",
      "100%|██████████| 663924/663924 [00:12<00:00, 51449.98it/s]\n",
      "100%|██████████| 663924/663924 [00:12<00:00, 51354.61it/s]\n",
      "100%|██████████| 663924/663924 [00:12<00:00, 51111.63it/s]\n",
      "100%|██████████| 663924/663924 [00:13<00:00, 50258.59it/s]\n",
      "100%|██████████| 663924/663924 [00:15<00:00, 42069.27it/s]\n",
      "100%|██████████| 663924/663924 [00:16<00:00, 40421.82it/s]\n",
      "100%|██████████| 663924/663924 [00:16<00:00, 41137.84it/s]\n",
      "100%|██████████| 663924/663924 [00:15<00:00, 42248.62it/s]\n",
      "100%|██████████| 663924/663924 [00:17<00:00, 37032.16it/s]\n",
      "100%|██████████| 663924/663924 [00:46<00:00, 14134.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.6 s, sys: 6.49 s, total: 14.1 s\n",
      "Wall time: 58.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing(twitter, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.01 s, sys: 2.89 s, total: 7.9 s\n",
      "Wall time: 16.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "temp_vocab = list(set(cleaning.multiprocessing(twitter, cleaning.unique_words)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CPU times: user 372 ms, sys: 1.14 s, total: 1.51 s\n",
      "Wall time: 3.08 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.duplicate_dots_marks_exclamations, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 228909.94it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 243831.84it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 243182.61it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 241498.20it/s]\n",
      " 69%|██████▉   | 456653/663924 [00:02<00:01, 192979.20it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 240239.29it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 240022.07it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 225361.60it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 234829.92it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 14278.48it/s]177.36it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 234639.78it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 181412.11it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 175647.44it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 181968.87it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 187053.49it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 168902.47it/s]\n",
      "100%|██████████| 663924/663924 [00:11<00:00, 59348.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.12 s, sys: 5.07 s, total: 11.2 s\n",
      "Wall time: 21.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CPU times: user 344 ms, sys: 1.24 s, total: 1.59 s\n",
      "Wall time: 1.93 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.remove_underscore, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 241712.01it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 241071.46it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 244007.72it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 227009.21it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 244159.64it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 243877.37it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 231403.12it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 236164.67it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 243859.77it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 239785.03it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 13137.99it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 186504.91it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 183242.26it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 176249.01it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 188563.10it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 166933.58it/s]\n",
      "100%|██████████| 663924/663924 [00:11<00:00, 60246.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.13 s, sys: 5.45 s, total: 11.6 s\n",
      "Wall time: 21.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\n",
      "CPU times: user 313 ms, sys: 1.27 s, total: 1.58 s\n",
      "Wall time: 1.98 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.isolate_spamchars, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 232314.15it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 229604.51it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 234634.01it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 230789.21it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 216641.02it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 230093.74it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 230928.86it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 17791.32it/s]856.64it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 222489.22it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 232118.40it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 225265.65it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 171329.94it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 174392.26it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 179140.24it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 179355.25it/s]\n",
      "100%|██████████| 663924/663924 [00:04<00:00, 157705.46it/s]\n",
      "100%|██████████| 663924/663924 [00:11<00:00, 57841.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.16 s, sys: 5.05 s, total: 11.2 s\n",
      "Wall time: 21.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CPU times: user 303 ms, sys: 1.41 s, total: 1.72 s\n",
      "Wall time: 1.97 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.break_short_words, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 242455.10it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 235904.68it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 232367.77it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 232797.16it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 243893.13it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 244565.84it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 233388.69it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 226939.15it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 235302.07it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 15391.94it/s]267.40it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 239128.61it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 187529.55it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 178261.65it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 185390.82it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 182817.87it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 167764.92it/s]\n",
      "100%|██████████| 663924/663924 [00:11<00:00, 58795.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.3 s, sys: 4.94 s, total: 11.2 s\n",
      "Wall time: 21.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CPU times: user 317 ms, sys: 1.31 s, total: 1.63 s\n",
      "Wall time: 1.89 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.break_long_words, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 238566.20it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 242395.62it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 241976.62it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 231825.08it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 236952.98it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 239790.36it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 239655.27it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 13740.55it/s]785.48it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 209281.18it/s]\n",
      "100%|██████████| 663924/663924 [00:05<00:00, 122138.66it/s]\n",
      "100%|██████████| 663924/663924 [00:05<00:00, 115957.47it/s]\n",
      "100%|██████████| 663924/663924 [00:06<00:00, 99612.60it/s] \n",
      "100%|██████████| 663924/663924 [00:06<00:00, 97692.48it/s] \n",
      "100%|██████████| 663924/663924 [00:06<00:00, 99545.22it/s] \n",
      "100%|██████████| 663924/663924 [00:06<00:00, 100174.22it/s]\n",
      "100%|██████████| 663924/663924 [00:07<00:00, 93161.19it/s] \n",
      "100%|██████████| 663924/663924 [00:12<00:00, 53336.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.72 s, sys: 4.77 s, total: 11.5 s\n",
      "Wall time: 24.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CPU times: user 311 ms, sys: 1.39 s, total: 1.7 s\n",
      "Wall time: 1.92 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.remove_ending_underscore, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 243249.73it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 245374.33it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 245579.71it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 243485.60it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 235513.69it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 229653.56it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 236242.57it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 225331.13it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 238689.24it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 238139.12it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 18703.70it/s]568.36it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 182643.83it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 182712.45it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 185277.45it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 188918.11it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 168759.28it/s]\n",
      "100%|██████████| 663924/663924 [00:11<00:00, 60094.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.29 s, sys: 5.14 s, total: 11.4 s\n",
      "Wall time: 21.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CPU times: user 328 ms, sys: 1.29 s, total: 1.62 s\n",
      "Wall time: 1.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.remove_starting_underscore, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 239403.06it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 232249.77it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 238024.58it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 241556.23it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 241813.96it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 239428.86it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 227818.39it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 233877.73it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 18662.09it/s]727.70it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 228805.78it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 226999.16it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 185228.21it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 181408.99it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 185428.18it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 185761.23it/s]\n",
      "100%|██████████| 663924/663924 [00:04<00:00, 161548.67it/s]\n",
      "100%|██████████| 663924/663924 [00:11<00:00, 59968.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.16 s, sys: 4.72 s, total: 10.9 s\n",
      "Wall time: 21.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7786\n",
      "CPU times: user 352 ms, sys: 1.41 s, total: 1.76 s\n",
      "Wall time: 2.03 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.end_punct, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 224379.14it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 224690.32it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 228376.13it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 220619.87it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 213269.87it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 220095.17it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 226333.66it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 2623.08it/s]1443.85it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 215592.00it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 228138.16it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 219233.91it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 166275.33it/s]\n",
      "100%|██████████| 663924/663924 [00:04<00:00, 164746.29it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 170125.66it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 171471.82it/s]\n",
      "100%|██████████| 663924/663924 [00:04<00:00, 150967.02it/s]\n",
      "100%|██████████| 663924/663924 [00:11<00:00, 55675.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.15 s, sys: 4.88 s, total: 11 s\n",
      "Wall time: 22.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8141\n",
      "CPU times: user 368 ms, sys: 1.3 s, total: 1.66 s\n",
      "Wall time: 1.93 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.start_punct, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 224634.30it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 222630.26it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 209072.31it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 212386.44it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 219881.51it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 222065.32it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 13368.30it/s]158.24it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 222114.59it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 209450.49it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 207481.18it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 215685.53it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 170703.71it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 166547.76it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 168638.95it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 169820.88it/s]\n",
      "100%|██████████| 663924/663924 [00:04<00:00, 149467.66it/s]\n",
      "100%|██████████| 663924/663924 [00:12<00:00, 54774.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.9 s, sys: 4.8 s, total: 10.7 s\n",
      "Wall time: 21.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CPU times: user 320 ms, sys: 1.42 s, total: 1.74 s\n",
      "Wall time: 2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.join_dashes, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 663924/663924 [00:02<00:00, 239986.26it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 243900.27it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 236172.18it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 229942.61it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 238013.94it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 238612.57it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 227962.64it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 226272.23it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 232656.96it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 17367.72it/s]372.32it/s]\n",
      "100%|██████████| 663924/663924 [00:02<00:00, 226539.16it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 186940.01it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 182148.29it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 184634.49it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 185687.41it/s]\n",
      "100%|██████████| 663924/663924 [00:03<00:00, 168506.39it/s]\n",
      "100%|██████████| 663924/663924 [00:11<00:00, 58428.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.08 s, sys: 4.87 s, total: 10.9 s\n",
      "Wall time: 21.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter = cleaning.multiprocessing_multiple(twitter, temp_dict, cleaning.string_dict_cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter = [s.lower() for s in twitter]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ternyata kl lg sdih bisa ngasilin makanan enak',\n",
       " 'abu kampret',\n",
       " 'bapa saya suka pake oppo saya sukanya nokia kaka saya sukanya samsung yg penting punya hape aja',\n",
       " 'ngelamar kasih cincin tp kok mukanya songong ya sedih gue liatnya',\n",
       " 'caption iki nggarai uwong males nikah min ya kali manusia arep punah ngunu neg gak nikah',\n",
       " 'pertanyaannya sederhana jika kami memang dukung prabowo ngapain selama kampanye kemarin capek dukung jokowi sampa',\n",
       " '',\n",
       " 'memiliki sedikit iman lebih berharga dari pada memiliki segudang emas',\n",
       " 'untuk mengamankan suara partai ahmad rofiq selaku sekjen partai perindo meminta kepada seluruh caleg dan struktur',\n",
       " 'dom jakpus sih bebas mau ketemuan or shopee',\n",
       " 'bisa dapet duit ini kaga punya mobil juga kan kaya gemer gemer ini kaga',\n",
       " 'on jadahnyaaa in sorry bad english hihuheheho',\n",
       " 'valentino rossi tidak setuju kompetisi motogp dimulai dari eropa',\n",
       " 'sis tak faham apa yang mungkin ini puncanya tu',\n",
       " 'martabak terang bulan martabak untuk yg asin gurih a k a martabak telor terang bulan untuk yg manis yg gw s',\n",
       " 'dia dah tua put dah nak plus dia tak start regularly kat man utd so mesti ka',\n",
       " 'sejarah susah',\n",
       " 'loop in nama dlm email pon boleh jd issue dah org email aku reply all jelaaa ade mase pulak aku nak tengok satu nama recipients',\n",
       " 'tak sakit pun tapi saja nak bau minyak freshcare sbb bau lavender',\n",
       " 'rosmah',\n",
       " 'bila kau tengah feeling lagu raya',\n",
       " 'kekasih bayangan',\n",
       " 'hidup ni jgn terlalu nk mendongak ke atas nanti jatuh padan muka kau',\n",
       " 'pak kim',\n",
       " 'di rumah ga liat pohon kelapa sama nanas kan apalagi pohon pisang',\n",
       " 'wkwk',\n",
       " 'kanan sja bu',\n",
       " 'tak pon sebelum masuk dapur bagi salam dulu kan molek gitu',\n",
       " 'hilang nyawaku aku tgk',\n",
       " 'masuk ke channel bang evan ke ni',\n",
       " 'yg minat saya pon bole lekk',\n",
       " 'yer lah sbb sombong mmg lah',\n",
       " 'nti aku tengok dulu tiket dari kl pukul berapa ada nahh',\n",
       " 'ni pukul berapa tah nak sampai ukm tetiba jalan tutup pulak kena lalu jalan jauh',\n",
       " 'tkpe hehe asalkan effort ada',\n",
       " 'kenapa kipas number pun sejuk kalau bilik aku sorang ni aku dah tutup',\n",
       " 'google cabut lisensi android huawei bagaimana nasib honor tekno',\n",
       " 'dari semalam tak tidur lagi ek ni kejap lagi jangan leceh nampak tilam bantal confirm nyenyak punya',\n",
       " 'tidur di ubin biar ga jatoh lg',\n",
       " 'guys tolong rt tweet ni sampai owner dia dapat phone ni tertinggal kat belakang teksi pakcik saya model oppo r s',\n",
       " 'jujur kacang ijo',\n",
       " 'sahur tengah malam kaya nya enak ya',\n",
       " 'jenis jenis orang stalking di media sosial pakai akun palsu pakai akun temannya sanak saudaranya handai tau',\n",
       " 'benersi ga buka sm yg minyak aka gorengan amp makan nasi tapi abis pudding setengah lingkaran makannya mi trs ishy',\n",
       " 'hahahahha bahaya bela kucing comel ni sebab nnti hilang kena curi',\n",
       " 'aku ada motor racing aku bawa ronda awek lu bonceng sedar sedar seluat tkde dah punca mat rempit takboleh rap',\n",
       " 'pak prabowo itu vibesnya kebun binatang banget ya peliharaannya kucing sukanya naek kuda kemana mana pake baju safari',\n",
       " 'nak happiness bkn pegih ngn laki lain happiness it s between u and me bedek uh kau ckp takda happiness ss sem',\n",
       " 'makan serabi enak pas lagi panas serabinya terbuat dari kelapa neng tasya aa ikhlas',\n",
       " 'siapapun orangnya meski dia ustadz bersorban dan berjubah putih klo sdh k',\n",
       " 'loh kenapa kan marga oppa juga lee pasti enak yaudah oppa jalan jalan dong biar bisa liat pemandangan',\n",
       " 'ada apa yaa mbak mbak plat ag inii',\n",
       " 'kanan',\n",
       " 'nak mee kari nak sate nak laksa nak bihun sup nakkkkk semuaaaaa',\n",
       " 'jadi lumba lumba',\n",
       " 'still sedihbgt kebayang kan betapa sedih lu gak tau gimana lu di waktu tahun tahun thn dan seteru',\n",
       " 'iyaaa gue di hima periode ditambah malamnya gue rapat atau latihan ukm jadi kalo mau nongkrong bisanya jam keatas',\n",
       " 'bangga manfaat dilan perputaran uang yg mendukung pertumbuhan ekonomi mikro makro mengurangi pengangguran',\n",
       " 'drpd lahir sampai sekarang aku asyik ngantuk je',\n",
       " 'pgn chatime xixi tp jauh',\n",
       " 'dah tak kasi lampu ijo loh tinggal pepet to cuk hwhw',\n",
       " 'kecewa',\n",
       " 'batok kelapa menjadi bara terbakar semua tidak tersisa wahai saudara seiman senegara saya ucapakan selamat puas',\n",
       " 'jy liner jgak ke',\n",
       " 'ada benda mcm kotor mcm air susu atas kereta mcm ada org campak mula ingat mcm taik burung tp lain mcm',\n",
       " 'air koroi',\n",
       " 'ilmu perpustakaan point kuliah ttg manajemen perpustakaan literasi informasi teknologi informasi',\n",
       " 'ajax spurs lah anti menstrim',\n",
       " 'abis telan biji durian kali',\n",
       " 'apaan rambut item',\n",
       " 'senin april kita memperingati hari bumi bumi kita saat ini lagi menjerit kesakitan karena dirusak untuk m',\n",
       " 'gaya hidup sihat delayed',\n",
       " 'lia pulang mereka semuanya pedo kecuali aku jangan mau',\n",
       " 'bangun lambat lepas tu jalan jem gile haihhhh so stressss',\n",
       " 'nice igstory harini dah tak nmpak org repost sudan meal project tu',\n",
       " 'gone apa gitu je laa sendu sorang',\n",
       " 'bukan pola pikir seorang profesor hukum tapi cara berpikir seorang pedagang cendol',\n",
       " 'sobatani sebagai upaya meningkatkan generasi petani kementan membuat terobosan dengan mengubah sekolah tinggi pen',\n",
       " 'beomgyu ngambilin confetti yang nyangkut di rambut jimin dong liat gini aja soft akutuh cha',\n",
       " 'eh hello bosan tu sbb kau xmenghayati hahaha',\n",
       " 'crash on lebuhraya damansara puchong putrajaya amp cyberjaya still delaying traffic m more than usual',\n",
       " 'waduuh kamu dengerinnya sambil minum',\n",
       " 'rasa rasanya kalo lg gapunya duit gini nemu duit recehan yang nyelip dikantong celana atau nemu duit kerincingan',\n",
       " 'anjing lagi having sex gitu kak',\n",
       " 'i m at csf computer exchange cx in cyberjaya selangor darul ehsan w',\n",
       " 'clip percutian yang menarik haruslah dipadankan dengan tempat rare dan istimewa berlatar belakangkan gunung santubong dan berhadapan dengan laut china selatan oh indahnya dunia jom follow instagram kami',\n",
       " 'alhamdulillah hari ni iftar nasi kerabu ayam madu kak yong n laksam buat kali terakhir sebab kak aini last da berju',\n",
       " 'saya udah sering banget ngadepin jalanan macet di jakarta tapi sejauh ini yg paling anjing sih semuanya',\n",
       " 'hi baby baru bangun baby emo',\n",
       " 'twitter please do ur magic ini pertama kali nyah gua ngajak jalan dia karna selama bertahun tahun dia kuliah di j',\n",
       " 'jum cuckoo bersama nabil ahmad',\n",
       " 'nikammy',\n",
       " 'resort datuk jhon gani kuala penyu boleh bawa keluarga santai saja tempatnya pantai nya bersih dan indah tenan',\n",
       " 'uni kenapa sistem masuk sekolah ke tingkat lanjutan terlalu susah skr in',\n",
       " 'krisis perlembagaan kedua bermula balik dgn orang sama dgn',\n",
       " 'bagi saya diusia an kata jahat bukan lagi sesederhana mainan yang dirampas atau buku pr yang dirobek teman j',\n",
       " 'lapor arah demak tersendat dari tambak lorok wib dan sekarang di terminal terboyo masih rendet',\n",
       " 'kph ujung tombak pendukung visi misi gubernur kalbar dengan mengoptimalkan tugas dan fungsi pokok kesatuan peman',\n",
       " 'aku sedih ni tak ada siapa nak hiburkan ke',\n",
       " 'gue baru bangun juga lagi males pergi mana rumah gue kek kapal pecah utg kaga main twitter tmn gue zwoakowka kalo ga udh diciduk']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ms-socialmedia.json', 'w') as fopen:\n",
    "    json.dump(twitter, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://malaya-dataset.s3-ap-southeast-1.amazonaws.com/dumping/twitter/2020-02-22-twitter-dump-en.json\n",
    "# !wget https://malaya-dataset.s3-ap-southeast-1.amazonaws.com/dumping/twitter/2020-03-08-twitter-dump-en.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-22-twitter-dump-en.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 717858/717858 [00:00<00:00, 1297407.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-08-twitter-dump-en.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 716237/716237 [00:00<00:00, 1164060.63it/s]\n"
     ]
    }
   ],
   "source": [
    "files = ['2020-02-22-twitter-dump-en.json', '2020-03-08-twitter-dump-en.json']\n",
    "\n",
    "twitter_en = []\n",
    "\n",
    "for file in files:\n",
    "    print(file)\n",
    "\n",
    "    with open(file) as fopen:\n",
    "        temp = json.load(fopen)\n",
    "\n",
    "    for i in tqdm(range(len(temp))):\n",
    "        retweet = temp[i]['retweet_text_full']\n",
    "        t = temp[i]['data_text']\n",
    "        if retweet != 'NULL' and len(retweet) > len(t):\n",
    "            t = retweet\n",
    "        twitter_en.append(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 89630/89630 [00:01<00:00, 44923.69it/s]\n",
      "100%|██████████| 15/15 [00:00<00:00, 34174.12it/s]5it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 43152.08it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 43855.88it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 44633.34it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 43566.05it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 43143.04it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 44134.41it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 44112.90it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 42944.40it/s]\n",
      "100%|██████████| 89630/89630 [00:01<00:00, 45453.43it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 43092.49it/s]\n",
      "100%|██████████| 89630/89630 [00:01<00:00, 45474.05it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 42875.53it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 38590.68it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 43295.89it/s]\n",
      "100%|██████████| 89630/89630 [00:02<00:00, 41277.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.04 s, sys: 1.81 s, total: 2.85 s\n",
      "Wall time: 4.77 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "twitter_en = cleaning.multiprocessing(twitter_en, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_en = [s.lower() for s in twitter_en]\n",
    "twitter_en = list(filter(None, twitter_en))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('en-socialmedia.json', 'w') as fopen:\n",
    "    json.dump(twitter_en, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
